package stackOverflow;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.sql.SQLException;
import java.util.ArrayList;
//imports for DEGUB!
import java.util.zip.GZIPInputStream;
import java.io.InputStreamReader;

import javax.xml.parsers.ParserConfigurationException;

import org.xml.sax.SAXException;

import stackOverflow.DBhandler.DB;


/**
 * @author Boris Cherkasky
 *
 */
public class StackOverFlowReader{
    private static final int MAX_REQUEST_PAGES = 1000;
    //json reader/parser
    private JsonHandler jHandler;
    //App key, used in the interaction with the StackOverflow API
    private String appKey = "LrTicNhUF0WBcvMaLldsSw";
    //Path to folder where results will be commited to
    private String resultsLocation;
    
    //Set DEBUG if connection to StackOverflow needs to be checked
    //The content of the input stream from StackOverflow will be printed to the console 
    private final static boolean DEBUG = false;
    
    //request types this application supports
    //SEARCH 		- a search query, used to fetch questions ids
    //QUESTION_FETCH	- a question fetch query, used to fetch question body, meta data and question answers
    //ANSWER_FETCH	- an answer fetch query, used to fetch answer body and meda
    enum RequestType{
	SEARCH, QUESTION_FETCH, ANSWER_FETCH
    }
    
    public StackOverFlowReader(String path){
	this.resultsLocation = path;
    }
    
    /**
     * Make a search query to StackOverflow with maximum result range and get the total
     * number of available results
     * @param query
     * @return total number of available results
     */
    private int getNumberOfAvailableResults(String query){
	createJsonStream(RequestType.SEARCH, query, MAX_REQUEST_PAGES, 0);
	return jHandler.getNumberOfResultObjects();
    }
    
    /**
     * Create a json stream reader object according to the query type and parameters
     * @param type
     * @param query
     * @param pageNum
     * @param id
     */
    private void createJsonStream(RequestType type, String query, int pageNum, int id){
	URL stackOverflow = null;
	try {
	    switch (type){
	    	case SEARCH:
	    	    //replace whitespaces in the query with "+"
	    	    query = query.replace(" ", "+");
//	    	    String tagged = "java";
	    	    String tagged = query.toLowerCase().replace(" ", "");
	    	    String notTaged = "c%2B%2B%3Bphp%3Bc%3Bpython%3Bruby%3Bjavascript%3Bc%23%3B.net";
	    	    stackOverflow = new URL("http://api.stackoverflow.com/1.0/search?intitle="
	    		    + query + "&pagesize=100&tagged=" + tagged + "&nottagged=" 
	    		    + notTaged + "&key=" + appKey);
	    	    break;
	    	case QUESTION_FETCH:
	    	    stackOverflow = new URL("http://api.stackoverflow.com/1.0/questions/" 
	    		    		 	+ id + "?body=true&id="+ id +"&pagesize=100&page=" + pageNum
	    		    		 	+ "&key=" + appKey);
	    	    break;
	    	case ANSWER_FETCH:
        	    stackOverflow = new URL("http://api.stackoverflow.com/1.0/questions/" + id + 
        		    			"/answers?body=true&id=" + id + "&pagesize=100&page=" + pageNum
        		    			+ "&key=" + appKey); 
	    	    break;
	    }
	    
	    InputStream inputStream = stackOverflow.openStream();
///////////////////////////DEBUG////////////////////////////////
//	    *********print stream content*********
	    if (DEBUG){
		InputStreamReader isr = new InputStreamReader(new GZIPInputStream(inputStream), "utf-8");
		int val = 0;
		while (val != -1){
		    val = isr.read();
		    System.out.print((char)val);
		}
		System.exit(0);
	    }
//	    *********print stream content*********
////////////////////////////////////////////////////////////////
	    jHandler = new JsonHandler(inputStream);
	} catch (MalformedURLException e) {
	    e.printStackTrace();
	} catch (IOException e) {
	    e.printStackTrace();
	}	
	return;
    }
    
    /**
     * send a search for {query} request to StackOverflow
     * @param query
     * @param NumberOfPages = -1 if unbounded
     * @param fromPage		- the first page to be fetched
     * @return ArrayList<CodeObject> 
     */
    public ArrayList<CodeObject> searchStackverflow(String query, int fromPage, int NumberOfPages){    
	//check parameters
	fromPage = (fromPage <= 0 ? 1 : fromPage);
	NumberOfPages = (NumberOfPages <= 0 ? StackOverFlowReader.MAX_REQUEST_PAGES : NumberOfPages);
	int resultNum = -1 , pageNum = fromPage;
	boolean isFirstIeration = true;
	ArrayList<CodeObject> searchResults = new ArrayList<CodeObject>();
	do {
	    //create one more stream
	    System.out.println("--Search request for page #" + pageNum);
	    createJsonStream(RequestType.SEARCH, query, pageNum, 0);	//get stream
	    if (isFirstIeration){
		//get Total number of results
		resultNum = jHandler.getNumberOfResultObjects();
		isFirstIeration = false;
	    }
	    //extract codeOcjects from search results 
	    System.out.println("--Extracting page #" + pageNum);
	    searchResults.addAll(jHandler.searchResult());
	    
	    resultNum -= 100;
	    pageNum ++;
	} while (resultNum > 0 && pageNum < fromPage + NumberOfPages);
	return searchResults;
    }
    
    /**
     * get code samples from question with id {questionID}
     * @param qustionID
     * @return
     */
    public ArrayList<String> fetchQuestionCodeSamples(int qustionID){
	createJsonStream(RequestType.QUESTION_FETCH, null, 1, qustionID);
	String questionBody = jHandler.questionCodeSamples();
	ArrayList<String> questionCodeSamples = CodeHandler.extractCode(questionBody);
	
	return questionCodeSamples;
    }
    
    
    /**
     * get question answers as AnswerObjects
     * @param questionID
     * @return
     */
    public ArrayList<AnswerObject> fetchQuestionAnsers(int questionID){
	createJsonStream(RequestType.ANSWER_FETCH, null, 1, questionID);
	ArrayList<AnswerObject> answers = jHandler.questionAnswers();
	ArrayList<AnswerObject> emptyAnswers = new ArrayList<AnswerObject>();
	//extract code samples from each answer
	for (AnswerObject ans : answers) {
	    if (ans.getCode().size() == 0){
		//answer has no code samples - remove answer
		emptyAnswers.add(ans);
		continue;
	    }
	    ArrayList<String> answerCodeSamples = CodeHandler.extractCode(ans.getCode().get(0));
	    ans.getCode().remove(0);		//remove answer body from AnswerObject
	    ans.setCode(answerCodeSamples);	//set code samples to the AnswerObject
	}
	
	//remove empty answers
	for (AnswerObject ans : emptyAnswers) {
	    answers.remove(ans);
	}
	return answers ;
    }
    
    /**
     * query stackoverflow and extract code samples from the results
     * @param query
     * @param fromPage 	 - the first page to be fetched
     * @param numOfPages - the total number of pages to be fetched
     * @return ArrayList of {@link ResultObject}s
     */
    private ArrayList<ResultObject> fetchCodeObjects(String query, int fromPage, int numOfPages){
	ArrayList<ResultObject> results = new ArrayList<ResultObject>();
	
	//search stackoverflow
	System.out.println("===Searching...");
	ArrayList<CodeObject> questions = searchStackverflow(query, fromPage, numOfPages);
	System.out.println("||==============");
	System.out.println("|| Found " + questions.size() + " search results");
	System.out.println("||==============");
	System.out.println("===Done Searching...\n");
	System.out.println("===Extracting Code Samples and meta data...\n");
	//fill question code objects and get question answers
	int count = 1;
	for (CodeObject question : questions) {
	    //get question code samples
	    System.out.println("--Extracting question #" +count + " with ID #" + question.getId());
	    question.setCode(fetchQuestionCodeSamples(question.getId()));
	    //get question answers
	    ArrayList<AnswerObject> answers = fetchQuestionAnsers(question.getId());
	    
	    //create result object
	    ResultObject resObj = new ResultObject(question, answers);
	    results.add(resObj);
	    count++;
	}
	
	return results;
    }
    
    /**
     * Query DB for code objects. 
     * it's prerequired that the DB will be already set (by calling the loadDump method).<br/>
     * 
     * Posts table referenced as "p".<br/>
     * Users table referenced as "u".<br/>
     * 
     * @param where - the content of a WHERE statement (i.e. without the "WHERE" keyword).<br/>
     * The statement can contain any condition on any field of the users and posts talbe 
     * (the interesting ones are title and body).<br/>
     * To fetch ALL entries, set where parm to be null.<br/>
     * <i><b>Note:</b> I strongly recomend to work with the constants defined in {@link DB.Posts} and {@link DB.Users}
     * when writing querys to this method.</i>
     * 
     * @return 
     * @throws InstantiationException
     * @throws IllegalAccessException
     * @throws ClassNotFoundException
     * @throws SQLException
     * @throws ParserConfigurationException 
     * @throws IOException 
     * @throws SAXException 
     */
    private ArrayList<ResultObject> queryDump(String where) throws 
    	InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException {
	DBhandler db = new DBhandler();
	ArrayList<CodeObject> queryCodeResults = db.queryPosts(where);
	ArrayList<ResultObject> results = new ArrayList<ResultObject>();
	//convert queryCodeResults from CodeObject list to ResultObect list be setting an empty answers list
	for (CodeObject r : queryCodeResults) {
	    results.add(new ResultObject(r, new ArrayList<AnswerObject>()));
	}
	return results;
    }

    /**
     * Load dump data from xmls to a MySQL DB.
     * 
     * @param freshDB	- if set DB will be deleted
     * @param loadPosts - if set, posts will be loaded
     * @param loadUsers - if set, users will be loaded
     * @param loadVotes	- if set, votes will be loaded
     * @param postsXML	- path to posts.xml
     * @param userXML	- path to users.xml
     * @param votesXML	- path to votes.xml
     * @param loadOnlyJava - if set, only posts containing "java" tag will be loaded
     * @throws SQLException
     * @throws InstantiationException
     * @throws IllegalAccessException
     * @throws ClassNotFoundException
     * @throws SAXException
     * @throws IOException
     * @throws ParserConfigurationException
     */
    public void loadDump(boolean freshDB, boolean loadPosts, boolean loadUsers, boolean loadVotes,
	    		String postsXML, String userXML, String votesXML, boolean loadOnlyJava) 
    	throws SQLException, InstantiationException, IllegalAccessException, ClassNotFoundException, 
    		SAXException, IOException, ParserConfigurationException{
	
	DataLoader loader = new DataLoader(postsXML, userXML, votesXML);
	loader.loadDataDump(freshDB, loadPosts, loadUsers, loadVotes, loadOnlyJava);
    }
    
    /**
     * Filter bad code from results
     * 
     * @param results
     * @throws IOException
     */
    private void filterResults(ArrayList<ResultObject> results) throws IOException{
	CodeHandler ch = new CodeHandler(results);
	//filter bad code samples
	ch.filterBadCodeSamples();
    }
    
    /**
     * Write all results to files. 
     * files will be created in the folder that was defined on class instanciation.
     * 
     * @param results
     * @return the number of files created
     * @throws IOException
     */
    private int commitResults(ArrayList<ResultObject> results) throws IOException{
	CodeHandler ch = new CodeHandler(results);
	//write results to file
    	ch.showResults(resultsLocation);
    	return ch.getFilesCount();
    }
    
    /**
     * Search StackOverflow Web API for results.
     * Results will be written to the folder specified on class instanciation.
     * 
     * @param query
     * @param fromPage
     * @param pageNum
     * @return the number of files the were created 
     * @throws IOException
     */
    public int searchWebAPI(String query, int fromPage, int pageNum) throws IOException{
	System.out.println("==========");
	System.out.println("There are " + getNumberOfAvailableResults(query) + " available results at StackOverflow");
	System.out.println("==========");
	//Query the API
	ArrayList<ResultObject> results = fetchCodeObjects(query, fromPage, pageNum);
	//Filter bad code samples
	filterResults(results);
	//Commit results to disk
	return commitResults(results);
    }
    
    /**
     * Query local StackOverflow data dump for results
     * <i></i>
     * 
     * @param where
     * @return the number of files that where created
     * @throws InstantiationException
     * @throws IllegalAccessException
     * @throws ClassNotFoundException
     * @throws SQLException
     * @throws IOException
     */
    public int searchLocalDump(String where) throws InstantiationException, IllegalAccessException, 
    			ClassNotFoundException, SQLException, IOException{
	//Get results from dump
	ArrayList<ResultObject> results = queryDump(where);
	for (ResultObject r : results) {
	    //extract code samples from post body
	    ArrayList<String> objectCodeSamples = r.getQuestion().getCode();
	    ArrayList<String> code = CodeHandler.extractCode(objectCodeSamples.get(0));
	    objectCodeSamples.remove(0);
	    r.getQuestion().setCode(code);
	}
	//Filter bad code samples
//	filterResults(results); TODO - uncomment it!
	//Commit reults to disk
	return commitResults(results);
    }
    
///////////////////////////DEBUG////////////////////////////////
    /**
     * Parse a file - used for DEBUGing purposes
     *  
     * @param filePath
     * @return the content of the file as a string
     * @throws IOException
     */
    public static String parseFile(String filePath) throws IOException{
	FileReader fr = new FileReader(filePath);
	String content = "";
	int c;
	do {
	    c = fr.read();
	    if (c != -1){
		content = content.concat(String.valueOf((char)c));
	    }
	} while (c != -1);
	System.out.println(content);
	return content;
    }
////////////////////////////////////////////////////////////////
    
    /** 
     * @throws IOException 
     * @throws SQLException 
     * @throws ClassNotFoundException 
     * @throws IllegalAccessException 
     * @throws InstantiationException 
     * @throws ParserConfigurationException 
     * @throws SAXException 
     * 
     */
    public static void main(String[] args) throws IOException, InstantiationException, 
    		IllegalAccessException, ClassNotFoundException, SQLException, SAXException, ParserConfigurationException{
	int fromPage = 1;
	int pageNum = 1;
	
	
	
//	String query = "p."+DB.Posts.TITLE.value() + " LIKE \"%ubuntu%\"";	//Dump query
	String query = "ResultSet";						//Web API query
	String path = "C:\\Users\\Lithuanian\\Desktop\\test\\";
	System.out.println("Extract results for Query : \"" + query + "\"");
	StackOverFlowReader s = new StackOverFlowReader(path);
	s.loadDump(true, true, true, true, null, null, null, false);
	System.exit(0);
//	int n = s.searchLocalDump(query);					//Search Dump
	int n = s.searchWebAPI(query, fromPage, pageNum);			//Search WebAPI
	System.out.println("DONE, " + n + " files have been created!");
    }
}